package WordHandling;

import java.util.*;
import java.io.*;


public class Tokenizer implements project3.WordTokenizer {
		
		/**this option controls whether you are stemming a query or processing actual documents
		 * necessary so that queries will not be added to database when parsed*/
        int option=0;
        //String queryString="";
        /**Array to hold the parsed strings of a query*/
        ArrayList<String> queryStrings = new ArrayList<String>();
        /**Array to hold all files that are processed*/
        ArrayList<File> documentList= new ArrayList<File>();
        
        /**If option is initialized with a 1, means parse a query, otherwise parse files*/
        Tokenizer(){}
        Tokenizer(int opt){option=opt;}

        /**Array to hold all tokens after parsing files, grouped by file where every inner array corresponds to a file*/
        public List<List<String>> filesOfWords= new ArrayList<List<String>>(); 

        /**takes a directory and tokenizes all files contained in that directory
         * @param path directory from which to extract files to be tokenized
         * stores all tokenized data in <code>filesOfWords</code>
         * */
        public void Tokenize(File path){
        	//extract all files from the directory
        	File file[]= path.listFiles();
        	//for each file extract entire contents as a string and pass to the function tokenize
        	//store tokenized data in filesOfWords
        	for (File f: file) {
                   try{
                           if(f==null){}
                           else{
                        	   documentList.add(f);
                           Scanner sc = new Scanner (f);
                    sc.useDelimiter("\\Z");
                    if(sc.hasNext())
                                filesOfWords.add(tokenize(sc.next()));
                    sc.close();}
                   }
         
                   catch(FileNotFoundException e){
                    e.printStackTrace();
                   }
           }

        }
        
        
        /**
         * Tokenizes a string by removing punctuation and stop words and returns a list of tokens.
         * Removes standalone numbers, but if contains any other character keeps it (e.g. keeps 1/2/03)
         * Removes all punctuation from files and all punctuation from queries     
         * @param bodyOfText a string containing a body of ASCII text
         * @return a list of tokens (i.e. English words) in the order that they
         * appear in <code>bodyOfText</code>
         */
        public List<String> tokenize(String bodyOfText) {
        	//temporarily holds tokenized Strings
        	List<String> bodyOfWords = new ArrayList<String>();
           
        	Scanner sc = new Scanner (bodyOfText);
            while(sc.hasNext()){
                   String word = sc.next();
           
                   //change everything to lower case   
                   word = word.toLowerCase();  
           
           //preserving special cases involving numbers  ex. dates 
           //only removes stand alone numbers
           if(Character.isDigit(word.charAt(0))){
                   boolean is_number=true;
                   for(int i=0; i<word.length();i++){
                           if(Character.isDigit(word.charAt(i))){}
                           else{is_number=false;i=word.length();}
                   }
                   //if parsing a query add the token to queryStrings
                   //if parsing a file add the token to bodyOfWords;
                   if(option==0 && is_number==false)
                           bodyOfWords.add(word);
                   else if(is_number==false){queryStrings.add(word);} 
           }
           else{
        	   //if not a special number case, remove all punctuation
        	   word = word.replaceAll("[^\\p{L}]", "");

         //remove stop words  
           if(word.equals("and")||
                           word.equals("or")||
                           word.equals("the")||
                           word.equals("a")||
                           word.equals("an")||
                           word.equals("if")||
                           word.equals("in")||
                           word.equals("not")||
                           word.equals("but")||
                           word.equals("to")||
                           word.equals("on")||
                           word.equals("of")||
                           word.equals("from")||
                           word.equals("for")||
                           word.equals("this")||
                           word.equals("that")||
                           word.equals("where")||
                           word.equals("when")||
                           word.equals("what")||
                           word.equals("how")||
                           word.equals("me")||
                           word.equals("i")||
                           word.equals("you")||
                           word.equals("they")||
                           word.equals("them")||
                           word.equals("us"))
                {/*do nothing with these words and therefore effectively remove them from the text*/}
          else{
                 if(option==0){
                  if(!word.equals(""))
                	 bodyOfWords.add(word);}
                 else {
                	 if(!word.equals(""))
                	 queryStrings.add(word);} 
          }
           }
           }
                return bodyOfWords;
        } 
        
        /**@ return all tokenized files, essentially a list of lists of strings where each inner
         * list represents a files containing all Tokens from that file*/
        public List<List<String>> getFiles(){
                return filesOfWords;
        }
        
        /**@return a list of all files that were tokenized*/
        public ArrayList<File> getDocumentList(){
        	return documentList;
        }
        
    /*    public String getQueryString(){
                return queryString;
        }*/
        
        /**@return a list of Strings, the parsed query*/
        public ArrayList<String> getQueryStrings(){
                return queryStrings;
        }
        
}
